package com.xinziruo.richclient.handler;

import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.mina.core.service.IoHandlerAdapter;
import org.apache.mina.core.session.IoSession;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.codehaus.jackson.JsonNode;
import org.htmlcleaner.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.xinziruo.richclient.engine.CrawlEngine;
import com.xinziruo.richclient.engine.CrawlResponse;
import com.xinziruo.richclient.engine.IntrudeCrawlEngine;
import com.xinziruo.richclient.engine.Param;
import com.xinziruo.richclient.engine.ParamType;
import com.xinziruo.richclient.engine.PayloadSource;
import com.xinziruo.richclient.engine.impl.HttpClientCrawlEngine;
import com.xinziruo.richclient.engine.impl.NumbersPayloadSource;
import com.xinziruo.richclient.engine.impl.StringListPayloadSource;
import com.xinziruo.richclient.util.ExceptionUtils;
import com.xinziruo.richclient.util.HTMLDecoder;
import com.xinziruo.richclient.util.JsonUtils;
import com.xinziruo.richclient.util.ShortLinkUtils;
import com.xinziruo.richclient.util.XPathUtils;
import com.xinziruo.richclient.support.command.Command;


/**
 * 
 * @author loudyn
 * @modify ajian
 */
public class ClientCommandProtocolHandlerForSolr extends IoHandlerAdapter {
	private static Logger LOGGER = LoggerFactory.getLogger(ClientCommandProtocolHandlerForSolr.class);
	private static final int MAX_TAGS_LENGTH = 20;


	private final Analyzer analyzer = new IKAnalyzer(true);
	
	private ConcurrentHashMap<String, CommonsHttpSolrServer> servers = new ConcurrentHashMap<String, CommonsHttpSolrServer>();

	@Override
	public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
		LOGGER.error(cause.getMessage());
		System.exit(0);
	}

	@Override
	public void messageReceived(IoSession session, Object message) throws Exception {

		Command comm = (Command) message;
		JsonNode json = JsonUtils.fromJsonString(new String(comm.content(), "utf-8"), JsonNode.class);
		System.out.println("recv server crawl command : " + json.path("name"));
		List<Param> params = readParams(json);
		String host = json.path("enterUrl").getTextValue();

		if (params.isEmpty()) {
			try {
				CrawlEngine engine = new HttpClientCrawlEngine(host);
				CrawlResponse response = engine.issueRequest();
				Document doc = CrawlParser.parse(response);
				handleDocument(doc, json);
			} catch (Exception e) {
				LOGGER.error(e.getMessage(),e);
			}

			return;

		}

		PayloadSource payload = readPayload(json);
		IntrudeCrawlEngine engine = new HttpClientCrawlEngine(host, params, payload);
		while (engine.hasNextRequest()) {
			try {

				CrawlResponse response = engine.issueRequest();
				Document doc = CrawlParser.parse(response);
				handleDocument(doc, json);

			} catch (Exception e) {
				LOGGER.error(e.getMessage(),e);
			}
		}

	}

	protected void handleDocument(Document doc, JsonNode json) throws Exception {

		NodeList items = XPathUtils.evalAsNodeList(doc, json.path("itemsExpression").getTextValue());
		if (items.getLength() <= 0) {
			return;
		}

		boolean isFollowItem = json.path("followingItem").getBooleanValue();
		if (isFollowItem) {
			followItem(items, json);
			return;
		}

		nofollowItem(items, json);
	}

	protected void nofollowItem(NodeList items, JsonNode json) {

		String sitename = json.path("name").getTextValue().split("-")[0];
		String classify = json.path("classify").getTextValue();
		String project = json.path("project").getTextValue();
		String enterUrl = json.path("enterUrl").getTextValue();
		List<SolrInputDocument> solrDocuments = new ArrayList<SolrInputDocument>();
		for (int i = 0; i < items.getLength(); i++) {
			Node item = items.item(i);
			Map<String, Object> values = CrawlParser.parse(enterUrl, item, json);
			try{SolrInputDocument solrDocument = createSolrInputDocument(values, sitename,classify,project,"");
				solrDocuments.add(solrDocument);
			}catch(RuntimeException e){
				//ignore this exception
			}
			
		}

		submitSolrInputDocuments(json, solrDocuments);
	}

	private void submitSolrInputDocuments(JsonNode json, List<SolrInputDocument> solrDocuments) {
		
		if(solrDocuments.isEmpty()){
			LOGGER.warn("crawl nothing,is there some xpath error occur ? please check the xpath expression.");
			return;
		}

		try {

			SolrServer server = getSolrServer(json.path("callbackUrl").getTextValue());
			server.add(solrDocuments);
			server.commit();
		} catch (Exception e) {
			throw ExceptionUtils.toUnchecked(e);
		}
	}

	private SolrServer getSolrServer(String callbackUrl) throws Exception {
		
		CommonsHttpSolrServer server = new CommonsHttpSolrServer(callbackUrl);
		server.setMaxRetries(2);
		server.setDefaultMaxConnectionsPerHost(30);
		server.setConnectionTimeout(3000);
		server.setMaxTotalConnections(30);
		
		CommonsHttpSolrServer newServer = servers.putIfAbsent(callbackUrl, server);
		if( null == newServer){
			newServer = server;
		}
		return newServer;
	}

	protected void followItem(NodeList items, JsonNode json) {

		String sitename = json.path("name").getTextValue().split("-")[0];
		String classify = json.path("classify").getTextValue();
		String project = json.path("project").getTextValue();
		String enterUrl = json.path("enterUrl").getTextValue();
		List<SolrInputDocument> solrDocuments = new ArrayList<SolrInputDocument>();
		for (int i = 0; i < items.getLength(); i++) {

			try {

				Node item = items.item(i);
				String locExpression = findLocExpression(json,"loc");
				String loc = XPathUtils.evalAsString(item, locExpression);
				loc = Utils.fullUrl(enterUrl, filterLocValue(loc,json,"loc"));
				String tconExpression = findLocExpression(json,"tcon");	//增加标签
				String tcon = StringUtils.join(XPathUtils.evalAsStringList(item, tconExpression), ";");
				CrawlEngine engine = new HttpClientCrawlEngine(loc);
				CrawlResponse response = engine.issueRequest();
				Document itemDoc = CrawlParser.parse(response);
				
				Map<String, Object> values = CrawlParser.parse(loc, itemDoc, json);
				replaceLocValue(values,loc);
				SolrInputDocument solrDocument = createSolrInputDocument(values, sitename,classify,project,tcon);
				solrDocuments.add(solrDocument);
			} catch (Exception e) {
				LOGGER.warn("error ocurr when follow item loc", e);
			}

		}

		submitSolrInputDocuments(json, solrDocuments);
	}

	private String filterLocValue(String loc, JsonNode json,String nodeName) {
		JsonNode node = findLocJsonNode(json,nodeName);
		
		if(null != node){
			
			for (JsonNode stringFilter : node.path("stringFilterConfigs")) {
				
				String regex = stringFilter.path("regex").getTextValue();
				String replacement = stringFilter.path("replacement").getTextValue();
				loc = loc.replaceAll(regex,replacement);
			}

		}
		
		return loc;
	}

	private JsonNode findLocJsonNode(JsonNode json,String nodeName) {
		for(JsonNode node : json.path("parseExpressions")){
			String name = node.path("name").getTextValue();
			
			if(StringUtils.equalsIgnoreCase(nodeName, name)){
				return node;
			}
		}
		
		return null;
	}

	private String findLocExpression(JsonNode json,String nodeName) {
		JsonNode locNode = findLocJsonNode(json,nodeName);
		if(null != locNode){
			return locNode.path("expression").getTextValue();
		}
		
		return null;
	}

	private void replaceLocValue(Map<String, Object> values, String loc) {
		if(values.containsKey("loc")){
			values.remove("loc");
		}
		
		values.put("loc", loc);
	}

	private SolrInputDocument createSolrInputDocument(Map<String, Object> values, String sitename,String classify,String project,String tcon) {
		String id = "";
		String tags = "";
		String price = "";
		SolrInputDocument solrDocument = new SolrInputDocument();
		for (Entry<String, Object> entry : values.entrySet()) {
			String fieldName = entry.getKey();
			Object fieldValue = entry.getValue();
			if (StringUtils.equalsIgnoreCase("price", fieldName)) {
 				if(fieldValue.toString().matches("[0-9]+(?:\\.[0-9]*)?")){
 					price = fieldValue.toString();
 				}else{
					price = "0.00";
				}
 				continue;
  			}
			else if (StringUtils.equalsIgnoreCase("tcon", fieldName)) {
 				continue;
  			}
			else if (StringUtils.equalsIgnoreCase("loc", fieldName)) {
				id = ShortLinkUtils.shortUrl(fieldValue.toString())[0];
			}
			else if(StringUtils.equalsIgnoreCase("title", fieldName)){
				tags = analyzeTitle(HTMLDecoder.decode(fieldValue.toString().trim()));
			}
			else if (StringUtils.equalsIgnoreCase("sitename", fieldName)) {

				if (StringUtils.isBlank(fieldValue.toString())) {
					fieldValue = sitename;
				}
			} 
			else if(fieldValue instanceof List<?>){		
				@SuppressWarnings("unchecked")
				List<String> fieldValueAsStringList = (List<String>) fieldValue;				
				fieldValue = StringUtils.join(removeEmptyValue(fieldValueAsStringList), ";");
			}

			solrDocument.addField(fieldName, HTMLDecoder.decode(fieldValue.toString()));
		}
		solrDocument.addField("tcon", tcon);
		solrDocument.addField("price", price);
		solrDocument.addField("oldprice", price);
        solrDocument.addField("id", id);       
 		solrDocument.addField("classify", classify);
 		solrDocument.addField("project", project);
		solrDocument.addField("idAsHash", hash(id));
		solrDocument.addField("tags", tags);
		solrDocument.addField("randomRegion", createRandomRegion());
		solrDocument.addField("createTime", System.currentTimeMillis());
		return solrDocument;
	}
	
	//移除集合中空的值
	private List<String> removeEmptyValue(List<String> fieldValueAsStringList) {
		List<String> temps = new ArrayList<String> ();
		if(fieldValueAsStringList.size() <= 0){
			return Collections.emptyList();
		}
		for(String temp : fieldValueAsStringList){
			if(!StringUtils.isBlank(temp.replaceAll("\n|\\s", ""))){
				temps.add(temp.replaceAll("\n|\\s", ""));
			}
		}
		return temps;				
	}

	public static class DocumentHelper {		
			public static  String getString(SolrInputDocument solrInputDocument, String key, String defaultValue) {
				if(!solrInputDocument.containsKey(key)||"".equals(solrInputDocument.getFieldValue(key))){
					return defaultValue;
				}
				return (String) solrInputDocument.getFieldValue(key);
			}
	}



	

	public enum DatePopulator {

		ME(Arrays.asList(new SimpleDateFormat("yyyy-MM-dd"),new SimpleDateFormat("yyyyMMdd"), new SimpleDateFormat("yyyy/MM/dd"))) {
			public long populate(String dateString) {

				for (SimpleDateFormat sdf : getFormats()) {
					try {
						
						return sdf.parse(dateString).getTime();
					} catch (Exception ingore) {
						// ingore it
					}
				}
				
				return 0;
			}
		};

		private final List<SimpleDateFormat> formats;

		private DatePopulator(List<SimpleDateFormat> formats) {
			this.formats = formats;
		}

		protected List<SimpleDateFormat> getFormats() {
			return formats;
		}

		public abstract long populate(String dateString);
	}
	
	
	private String analyzeTitle(String title) {
		StringBuilder buf = new StringBuilder();
		
		try{
			
			TokenStream token = analyzer.tokenStream(title, new StringReader(title));
			token.addAttribute(CharTermAttribute.class);
			
			while (token.incrementToken()) {
				CharTermAttribute termAttribute = token.getAttribute(CharTermAttribute.class);
				if(buf.length() > MAX_TAGS_LENGTH){
					break;
				}
				
				if(termAttribute.length() < 2){
					continue;
				}
				
				if(buf.length() > 0){
					buf.append(";");
				}
				
				buf.append(new String(termAttribute.buffer(), 0, termAttribute.length()));
			}
			
		}catch (Exception e) {
			throw ExceptionUtils.toUnchecked(e);
		}

		return removeUnlikeChar(buf.toString());
	}

	private static String removeUnlikeChar(String string) {
		return string.replaceAll("/", "");
	}
	
	private int hash(String id) {
		int h = id.hashCode();
		h ^= (h >>> 20) ^ (h >>> 12);
		return (h ^ (h >>> 7) ^ (h >>> 4)) & 0x7FFFFFFF;
	}

	private int createRandomRegion() {
		Random random = new Random();
		return random.nextInt(9000)+1000;
	}

	private List<Param> readParams(JsonNode json) {
		List<Param> result = new ArrayList<Param>();

		for (JsonNode node : json.path("params")) {

			Param param = new Param().setName(
											node.path("name").getTextValue()
									).setEncode(
											node.path("encode").getTextValue()
									).setType(
											ParamType.valueOf(node.path("type").getTextValue())
									).setAttack(
											node.path("attack").getBooleanValue()
									).setValue(
											node.path("value").getTextValue()
									).setClassify(
											node.path("classify").getTextValue()
									).setProject(
											node.path("project").getTextValue()
									);

			result.add(param);
		}

		return result;
	}

	private PayloadSource readPayload(JsonNode json) {
		String payload = json.path("payloadString").getTextValue();

		if (isNumberPayloadSource(payload)) {
			return asNumberPayloadSource(payload);
		}

		return new StringListPayloadSource(payload);
	}

	private PayloadSource asNumberPayloadSource(String payload) {
		String numberDescr = StringUtils.substringAfter(payload, ":");
		String[] numbers = numberDescr.split(",");

		return new NumbersPayloadSource(
										Integer.parseInt(numbers[0]),
										Integer.parseInt(numbers[1]),
										Integer.parseInt(numbers[2])
									);
	}

	private boolean isNumberPayloadSource(String payload) {
		if (StringUtils.isBlank(payload)) {
			return false;
		}

		return payload.startsWith("numbers");
	}
}
